<!DOCTYPE html><html lang="zh-CN" data-theme="light"><head><meta charset="UTF-8"><meta http-equiv="X-UA-Compatible" content="IE=edge"><meta name="viewport" content="width=device-width, initial-scale=1.0,viewport-fit=cover"><title>requests模块使用 | East'blog</title><meta name="author" content="侯玉东"><meta name="copyright" content="侯玉东"><meta name="format-detection" content="telephone=no"><meta name="theme-color" content="#ffffff"><meta name="description" content="requests模块使用">
<meta property="og:type" content="article">
<meta property="og:title" content="requests模块使用">
<meta property="og:url" content="https://houyudong1989.gitee.io/blog/2024/04/12/python/11%E3%80%81requests%E6%A8%A1%E5%9D%97%E4%BD%BF%E7%94%A8/index.html">
<meta property="og:site_name" content="East&#39;blog">
<meta property="og:description" content="requests模块使用">
<meta property="og:locale" content="zh_CN">
<meta property="og:image" content="https://houyudong1989.gitee.io/blog/img/library1.jpeg">
<meta property="article:published_time" content="2024-04-12T01:35:49.721Z">
<meta property="article:modified_time" content="2024-04-29T07:43:16.070Z">
<meta property="article:author" content="侯玉东">
<meta property="article:tag" content="requests模块使用">
<meta name="twitter:card" content="summary">
<meta name="twitter:image" content="https://houyudong1989.gitee.io/blog/img/library1.jpeg"><link rel="shortcut icon" href="/blog/img/head.jpg"><link rel="canonical" href="https://houyudong1989.gitee.io/blog/2024/04/12/python/11%E3%80%81requests%E6%A8%A1%E5%9D%97%E4%BD%BF%E7%94%A8/index.html"><link rel="preconnect" href="//cdn.jsdelivr.net"/><link rel="preconnect" href="//busuanzi.ibruce.info"/><link rel="stylesheet" href="/blog/css/index.css"><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@fortawesome/fontawesome-free/css/all.min.css" media="print" onload="this.media='all'"><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@fancyapps/ui/dist/fancybox/fancybox.min.css" media="print" onload="this.media='all'"><script>const GLOBAL_CONFIG = {
  root: '/blog/',
  algolia: undefined,
  localSearch: undefined,
  translate: undefined,
  noticeOutdate: undefined,
  highlight: {"plugin":"highlighjs","highlightCopy":true,"highlightLang":true,"highlightHeightLimit":false},
  copy: {
    success: '复制成功',
    error: '复制错误',
    noSupport: '浏览器不支持'
  },
  relativeDate: {
    homepage: false,
    post: false
  },
  runtime: '',
  dateSuffix: {
    just: '刚刚',
    min: '分钟前',
    hour: '小时前',
    day: '天前',
    month: '个月前'
  },
  copyright: undefined,
  lightbox: 'fancybox',
  Snackbar: undefined,
  infinitegrid: {
    js: 'https://cdn.jsdelivr.net/npm/@egjs/infinitegrid/dist/infinitegrid.min.js',
    buttonText: '加载更多'
  },
  isPhotoFigcaption: false,
  islazyload: false,
  isAnchor: false,
  percent: {
    toc: true,
    rightside: false,
  },
  autoDarkmode: false
}</script><script id="config-diff">var GLOBAL_CONFIG_SITE = {
  title: 'requests模块使用',
  isPost: true,
  isHome: false,
  isHighlightShrink: false,
  isToc: true,
  postUpdate: '2024-04-29 15:43:16'
}</script><script>(win=>{
      win.saveToLocal = {
        set: (key, value, ttl) => {
          if (ttl === 0) return
          const now = Date.now()
          const expiry = now + ttl * 86400000
          const item = {
            value,
            expiry
          }
          localStorage.setItem(key, JSON.stringify(item))
        },
      
        get: key => {
          const itemStr = localStorage.getItem(key)
      
          if (!itemStr) {
            return undefined
          }
          const item = JSON.parse(itemStr)
          const now = Date.now()
      
          if (now > item.expiry) {
            localStorage.removeItem(key)
            return undefined
          }
          return item.value
        }
      }
    
      win.getScript = (url, attr = {}) => new Promise((resolve, reject) => {
        const script = document.createElement('script')
        script.src = url
        script.async = true
        script.onerror = reject
        script.onload = script.onreadystatechange = function() {
          const loadState = this.readyState
          if (loadState && loadState !== 'loaded' && loadState !== 'complete') return
          script.onload = script.onreadystatechange = null
          resolve()
        }

        Object.keys(attr).forEach(key => {
          script.setAttribute(key, attr[key])
        })

        document.head.appendChild(script)
      })
    
      win.getCSS = (url, id = false) => new Promise((resolve, reject) => {
        const link = document.createElement('link')
        link.rel = 'stylesheet'
        link.href = url
        if (id) link.id = id
        link.onerror = reject
        link.onload = link.onreadystatechange = function() {
          const loadState = this.readyState
          if (loadState && loadState !== 'loaded' && loadState !== 'complete') return
          link.onload = link.onreadystatechange = null
          resolve()
        }
        document.head.appendChild(link)
      })
    
      win.activateDarkMode = () => {
        document.documentElement.setAttribute('data-theme', 'dark')
        if (document.querySelector('meta[name="theme-color"]') !== null) {
          document.querySelector('meta[name="theme-color"]').setAttribute('content', '#0d0d0d')
        }
      }
      win.activateLightMode = () => {
        document.documentElement.setAttribute('data-theme', 'light')
        if (document.querySelector('meta[name="theme-color"]') !== null) {
          document.querySelector('meta[name="theme-color"]').setAttribute('content', '#ffffff')
        }
      }
      const t = saveToLocal.get('theme')
    
        if (t === 'dark') activateDarkMode()
        else if (t === 'light') activateLightMode()
      
      const asideStatus = saveToLocal.get('aside-status')
      if (asideStatus !== undefined) {
        if (asideStatus === 'hide') {
          document.documentElement.classList.add('hide-aside')
        } else {
          document.documentElement.classList.remove('hide-aside')
        }
      }
    
      const detectApple = () => {
        if(/iPad|iPhone|iPod|Macintosh/.test(navigator.userAgent)){
          document.documentElement.classList.add('apple')
        }
      }
      detectApple()
    })(window)</script><meta name="generator" content="Hexo 6.3.0"></head><body><div id="loading-box"><div class="loading-left-bg"></div><div class="loading-right-bg"></div><div class="spinner-box"><div class="configure-border-1"><div class="configure-core"></div></div><div class="configure-border-2"><div class="configure-core"></div></div><div class="loading-word">加载中...</div></div></div><script>(()=>{
  const $loadingBox = document.getElementById('loading-box')
  const $body = document.body
  const preloader = {
    endLoading: () => {
      $body.style.overflow = ''
      $loadingBox.classList.add('loaded')
    },
    initLoading: () => {
      $body.style.overflow = 'hidden'
      $loadingBox.classList.remove('loaded')
    }
  }

  preloader.initLoading()
  window.addEventListener('load',() => { preloader.endLoading() })

  if (true) {
    document.addEventListener('pjax:send', () => { preloader.initLoading() })
    document.addEventListener('pjax:complete', () => { preloader.endLoading() })
  }
})()</script><div id="sidebar"><div id="menu-mask"></div><div id="sidebar-menus"><div class="avatar-img is-center"><img src="/blog/img/head.jpg" onerror="onerror=null;src='/img/friend_404.gif'" alt="avatar"/></div><div class="sidebar-site-data site-data is-center"><a href="/blog/archives/"><div class="headline">文章</div><div class="length-num">70</div></a><a href="/blog/tags/"><div class="headline">标签</div><div class="length-num">26</div></a><a href="/blog/categories/"><div class="headline">分类</div><div class="length-num">18</div></a></div><hr class="custom-hr"/><div class="menus_items"><div class="menus_item"><a class="site-page" href="/blog/"><i class="fa-fw fas fa-home"></i><span> 首页</span></a></div><div class="menus_item"><a class="site-page" href="/blog/archives/"><i class="fa-fw fas fa-archive"></i><span> 时间轴</span></a></div><div class="menus_item"><a class="site-page" href="/blog/tags/"><i class="fa-fw fas fa-tags"></i><span> 标签</span></a></div><div class="menus_item"><a class="site-page" href="/blog/categories/"><i class="fa-fw fas fa-folder-open"></i><span> 分类</span></a></div><div class="menus_item"><a class="site-page" href="/blog/link/"><i class="fa-fw fas fa-link"></i><span> 友联</span></a></div><div class="menus_item"><a class="site-page" href="/blog/about/"><i class="fa-fw fas fa-heart"></i><span> 关于</span></a></div></div></div></div><div class="post" id="body-wrap"><header class="post-bg fixed" id="page-header" style="background-image: url('/blog/img/library2.jpeg')"><nav id="nav"><span id="blog-info"><a href="/blog/" title="East'blog"><img class="site-icon" src="/blog/img/head.jpg"/><span class="site-name">East'blog</span></a></span><div id="menus"><div class="menus_items"><div class="menus_item"><a class="site-page" href="/blog/"><i class="fa-fw fas fa-home"></i><span> 首页</span></a></div><div class="menus_item"><a class="site-page" href="/blog/archives/"><i class="fa-fw fas fa-archive"></i><span> 时间轴</span></a></div><div class="menus_item"><a class="site-page" href="/blog/tags/"><i class="fa-fw fas fa-tags"></i><span> 标签</span></a></div><div class="menus_item"><a class="site-page" href="/blog/categories/"><i class="fa-fw fas fa-folder-open"></i><span> 分类</span></a></div><div class="menus_item"><a class="site-page" href="/blog/link/"><i class="fa-fw fas fa-link"></i><span> 友联</span></a></div><div class="menus_item"><a class="site-page" href="/blog/about/"><i class="fa-fw fas fa-heart"></i><span> 关于</span></a></div></div><div id="toggle-menu"><a class="site-page" href="javascript:void(0);"><i class="fas fa-bars fa-fw"></i></a></div></div></nav><div id="post-info"><h1 class="post-title">requests模块使用</h1><div id="post-meta"><div class="meta-firstline"><span class="post-meta-date"><i class="far fa-calendar-alt fa-fw post-meta-icon"></i><span class="post-meta-label">发表于</span><time class="post-meta-date-created" datetime="2024-04-12T01:35:49.721Z" title="发表于 2024-04-12 09:35:49">2024-04-12</time><span class="post-meta-separator">|</span><i class="fas fa-history fa-fw post-meta-icon"></i><span class="post-meta-label">更新于</span><time class="post-meta-date-updated" datetime="2024-04-29T07:43:16.070Z" title="更新于 2024-04-29 15:43:16">2024-04-29</time></span><span class="post-meta-categories"><span class="post-meta-separator">|</span><i class="fas fa-inbox fa-fw post-meta-icon"></i><a class="post-meta-categories" href="/blog/categories/python-%E7%88%AC%E8%99%AB/">python 爬虫</a></span></div><div class="meta-secondline"><span class="post-meta-separator">|</span><span class="post-meta-pv-cv" id="" data-flag-title="requests模块使用"><i class="far fa-eye fa-fw post-meta-icon"></i><span class="post-meta-label">阅读量:</span><span id="busuanzi_value_page_pv"><i class="fa-solid fa-spinner fa-spin"></i></span></span></div></div></div></header><main class="layout" id="content-inner"><div id="post"><article class="post-content" id="article-container"><h1 id="requests模块"><a href="#requests模块" class="headerlink" title="requests模块"></a>requests模块</h1><figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">requests是一个基于网络请求的模块,可以使用程序模拟浏览器上网。通过 requests 可以完成各种类型的 HTTP 请求，包括 HTTP、HTTPS、HTTP1.0、HTTP1.1 及各种请求方法。作为一个专门为「人类」编写的 HTTP 请求库，其易用性很强，因此在推出之后就迅速成为 Python 中首选的 HTTP 请求库。requests 库的最大特点是提供了简单易用的 API，让编程人员可以轻松地提高效率。</span><br></pre></td></tr></table></figure>

<h2 id="requests-安装"><a href="#requests-安装" class="headerlink" title="requests 安装"></a>requests 安装</h2><figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">pip install requests</span><br><span class="line">注意：requests依赖于python环境。</span><br></pre></td></tr></table></figure>

<h2 id="requests常用方法"><a href="#requests常用方法" class="headerlink" title="requests常用方法"></a>requests常用方法</h2><ul>
<li><strong><code>requests.get(url, params=None, headers=None, cookies=None, auth=None, timeout=None)</code></strong></li>
</ul>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">发送 GET 请求，并返回一个 Response 对象。url 参数是请求的 URL，params 是一个字典，包含请求的查询参数，headers 是一个字典，包含请求头信息，cookies 是一个字典，包含请求的 Cookie，auth 是一个元组，包含 HTTP 基本认证的用户名和密码，timeout 是超时时间。</span><br><span class="line">get请求的请求参数是跟在url后面的，url与参数？隔开，params封装的就是?后面的参数</span><br></pre></td></tr></table></figure>

<ul>
<li><strong><code>requests.post(url, data=None, json=None, headers=None, cookies=None, auth=None, timeout=None)</code></strong></li>
</ul>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">发送 POST 请求，并返回一个 Response 对象。data 是一个字典，包含 POST 请求的表单数据，json 是一个 JSON 对象，包含 POST 请求的 JSON 数据。</span><br><span class="line">data相当于get请求中的params，封装的是请求参数。</span><br></pre></td></tr></table></figure>

<ul>
<li><strong><code>requests.put(url, data=None, headers=None, cookies=None, auth=None, timeout=None)</code></strong></li>
</ul>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">requests.put(url, data=None, headers=None, cookies=None, auth=None, timeout=None)：发送 PUT 请求。</span><br></pre></td></tr></table></figure>

<ul>
<li><strong><code>requests.delete(url, headers=None, cookies=None, auth=None, timeout=None)</code></strong></li>
</ul>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">requests.delete(url, headers=None, cookies=None, auth=None, timeout=None)：发送 DELETE 请求。</span><br></pre></td></tr></table></figure>

<ul>
<li><strong><code>requests.head(url, headers=None, cookies=None, auth=None, timeout=None)</code></strong></li>
</ul>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">requests.head(url, headers=None, cookies=None, auth=None, timeout=None)：发送 HEAD 请求。</span><br></pre></td></tr></table></figure>

<ul>
<li><strong><code>requests.patch(url, data=None, headers=None, cookies=None, auth=None, timeout=None)</code></strong></li>
</ul>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">requests.patch(url, data=None, headers=None, cookies=None, auth=None, timeout=None)：发送 PATCH 请求。</span><br></pre></td></tr></table></figure>

<ul>
<li><strong><code>requests.options(url, headers=None, cookies=None, auth=None, timeout=None)</code></strong></li>
</ul>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">requests.options(url, headers=None, cookies=None, auth=None, timeout=None)：发送 OPTIONS 请求。</span><br></pre></td></tr></table></figure>

<h2 id="response-常用函数"><a href="#response-常用函数" class="headerlink" title="response 常用函数"></a>response 常用函数</h2><ul>
<li><strong><code>.status_code</code></strong></li>
</ul>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">获取响应的状态码，例如 `200` 表示请求成功，`404` 表示资源未找到，`500` 表示服务器内部错误等。</span><br></pre></td></tr></table></figure>

<ul>
<li><strong><code>.text</code></strong></li>
</ul>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">获取响应的文本内容，以字符串形式返回。</span><br></pre></td></tr></table></figure>

<ul>
<li><strong><code>.content</code></strong></li>
</ul>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">获取响应的内容，以字节形式返回。</span><br></pre></td></tr></table></figure>

<ul>
<li><strong><code>.json()</code></strong></li>
</ul>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">解析响应的 JSON 数据，并返回一个 Python 字典或列表。</span><br></pre></td></tr></table></figure>

<ul>
<li><strong><code>.headers</code></strong></li>
</ul>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">获取响应的头部信息，以字典形式返回。</span><br></pre></td></tr></table></figure>

<ul>
<li><strong><code>.cookies</code></strong></li>
</ul>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">获取响应中的 Cookie 信息，以字典形式返回。</span><br></pre></td></tr></table></figure>

<ul>
<li><strong><code>.url</code></strong></li>
</ul>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">获取响应的 URL，用于查看请求最终访问的地址，可用于处理重定向。</span><br></pre></td></tr></table></figure>

<ul>
<li><strong><code>.encoding</code></strong></li>
</ul>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">获取响应的编码方式。</span><br><span class="line"></span><br></pre></td></tr></table></figure>

<ul>
<li><strong><code>.raise_for_status()</code></strong></li>
</ul>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">如果响应状态码表示请求失败（如 4xx 或 5xx），则抛出异常，否则不做任何操作。</span><br></pre></td></tr></table></figure>

<ul>
<li><strong><code>.history</code></strong></li>
</ul>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">获取响应的历史记录，以列表形式返回，用于查看请求的重定向历史。</span><br></pre></td></tr></table></figure>

<h2 id="代理"><a href="#代理" class="headerlink" title="代理"></a>代理</h2><p><img src="1714375324124.png" alt="1714375324124"></p>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br></pre></td><td class="code"><pre><span class="line">用户访问外部网络时，将访问请求首先转发给代理服务器，然后再由代理服务器发起请求，这个过程被叫做网络代理访问。</span><br><span class="line">用户在家办公时想要访问公司内部网络，但由于公司内网网络不能通过互联网直接访问，就需要先访问到公司代理服务器，由代理服务器进行身份认证后，代替用户访问公司网络。</span><br></pre></td></tr></table></figure>

<blockquote>
<p><strong>在爬虫中为何需要使用代理？</strong></p>
</blockquote>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line">有些时候，需要对网站服务器发起高频的请求，网站的服务器会检测到这样的异常现象，则会讲请求对应机器的ip地址加入黑名单，则该ip再次发起的请求，网站服务器就不在受理，则我们就无法再次爬取该网站的数据。 </span><br><span class="line"></span><br><span class="line">使用代理后，网站服务器接收到的请求，最终是由代理服务器发起，网站服务器通过请求获取的ip就是代理服务器的ip，并不是我们客户端本身的ip。</span><br></pre></td></tr></table></figure>

<blockquote>
<p><strong>代理的匿名度</strong></p>
</blockquote>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br></pre></td><td class="code"><pre><span class="line">透明：网站的服务器知道你使用了代理，也知道你的真实ip。</span><br><span class="line"></span><br><span class="line">匿名：网站服务器知道你使用了代理，但是无法获知你真实的ip。</span><br><span class="line"></span><br><span class="line">高匿：网站服务器不知道你使用了代理，也不知道你的真实ip（推荐）。</span><br></pre></td></tr></table></figure>

<blockquote>
<p><strong>代理的类型（重要）</strong></p>
</blockquote>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br></pre></td><td class="code"><pre><span class="line"> http：该类型的代理服务器只可以转发http协议的请求。</span><br><span class="line"></span><br><span class="line">https：可以转发https协议的请求。</span><br></pre></td></tr></table></figure>

<blockquote>
<p><strong>如何获取代理?</strong></p>
</blockquote>
<figure class="highlight plaintext"><table><tr><td class="gutter"><pre><span class="line">1</span><br></pre></td><td class="code"><pre><span class="line">芝麻代理： [https://jahttp.zhimaruanjian.com/](https://jahttp.zhimaruanjian.com/)（推荐，有新人福利）</span><br></pre></td></tr></table></figure>

<h3 id="访问如下网址，返回自己本机ip"><a href="#访问如下网址，返回自己本机ip" class="headerlink" title="访问如下网址，返回自己本机ip"></a>访问如下网址，返回自己本机ip</h3><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">def</span> <span class="title function_">getSelfIp</span>():</span><br><span class="line">    web_url = <span class="string">&quot;http://www.cip.cc/&quot;</span></span><br><span class="line">    header = &#123;</span><br><span class="line">        <span class="string">&quot;User-Agent&quot;</span>: <span class="string">&quot;Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36&quot;</span>,</span><br><span class="line">    &#125;</span><br><span class="line">    response = requests.get(web_url,headers=header)</span><br><span class="line">    page_text = response.text</span><br><span class="line">    tree = etree.HTML(page_text)</span><br><span class="line">    text = tree.xpath(<span class="string">&#x27;/html/body/div/div/div[3]/pre/text()&#x27;</span>)[<span class="number">0</span>]</span><br><span class="line">    <span class="built_in">print</span>(text.split(<span class="string">&#x27;\n&#x27;</span>)[<span class="number">0</span>])</span><br></pre></td></tr></table></figure>
<h3 id="使用代理发起请求，查看是否可以返回代理服务器的ip"><a href="#使用代理发起请求，查看是否可以返回代理服务器的ip" class="headerlink" title="使用代理发起请求，查看是否可以返回代理服务器的ip"></a>使用代理发起请求，查看是否可以返回代理服务器的ip</h3><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br></pre></td><td class="code"><pre><span class="line"><span class="comment">#通过requests发起请求时，在请求参数proxies中填写代理ip和端口，实现代理</span></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">proxies</span>(): </span><br><span class="line">    headers = &#123;</span><br><span class="line">        <span class="string">&#x27;User-Agent&#x27;</span>: <span class="string">&#x27;Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.80 Safari/537.36&#x27;</span>,</span><br><span class="line">    &#125;</span><br><span class="line">    url = <span class="string">&#x27;http://www.cip.cc/&#x27;</span></span><br><span class="line"></span><br><span class="line">    page_text = requests.get(url, headers=headers, proxies=&#123;<span class="string">&#x27;http&#x27;</span>: <span class="string">&#x27;121.234.12.62:4246&#x27;</span>&#125;).text</span><br><span class="line">    tree = etree.HTML(page_text)</span><br><span class="line">    text = tree.xpath(<span class="string">&#x27;/html/body/div/div/div[3]/pre/text()&#x27;</span>)[<span class="number">0</span>]</span><br><span class="line">    <span class="built_in">print</span>(text.split(<span class="string">&#x27;\n&#x27;</span>)[<span class="number">0</span>])</span><br></pre></td></tr></table></figure>
<h3 id="构建代理池进行请求-需要申请三方代理池"><a href="#构建代理池进行请求-需要申请三方代理池" class="headerlink" title="构建代理池进行请求(需要申请三方代理池)"></a>构建代理池进行请求(需要申请三方代理池)</h3><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br><span class="line">16</span><br><span class="line">17</span><br><span class="line">18</span><br><span class="line">19</span><br><span class="line">20</span><br><span class="line">21</span><br><span class="line">22</span><br><span class="line">23</span><br><span class="line">24</span><br><span class="line">25</span><br><span class="line">26</span><br><span class="line">27</span><br><span class="line">28</span><br><span class="line">29</span><br><span class="line">30</span><br><span class="line">31</span><br><span class="line">32</span><br><span class="line">33</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">import</span> random</span><br><span class="line"><span class="keyword">import</span> time</span><br><span class="line"></span><br><span class="line"><span class="keyword">import</span> requests</span><br><span class="line"><span class="keyword">from</span> lxml <span class="keyword">import</span> etree</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">proxies2</span>():</span><br><span class="line">    headers = &#123;</span><br><span class="line">        <span class="string">&#x27;User-Agent&#x27;</span>: <span class="string">&#x27;Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/98.0.4758.80 Safari/537.36&#x27;</span>,</span><br><span class="line">    &#125;</span><br><span class="line">    <span class="comment"># 构建一个代理池</span></span><br><span class="line">    proxy_list = []</span><br><span class="line">    proxy_url = <span class="string">&#x27;http://webapi.http.zhimacangku.com/getip?neek=b090cde57620cd17&amp;num=2&amp;type=3&amp;time=4&amp;pro=0&amp;city=0&amp;yys=0&amp;port=11&amp;pack=0&amp;ts=0&amp;ys=0&amp;cs=0&amp;lb=2&amp;sb=&amp;pb=4&amp;mr=3&amp;regions=&#x27;</span></span><br><span class="line">    page_text = requests.get(url=proxy_url, headers=headers).text</span><br><span class="line">    <span class="keyword">for</span> ips <span class="keyword">in</span> page_text.split(<span class="string">&#x27;&lt;/br&gt;&#x27;</span>):</span><br><span class="line">        dic = &#123;&#125;</span><br><span class="line">        dic[<span class="string">&#x27;https&#x27;</span>] = ips</span><br><span class="line">        proxy_list.append(dic)</span><br><span class="line"></span><br><span class="line">    proxy_list = proxy_list[:-<span class="number">1</span>]</span><br><span class="line"></span><br><span class="line">    <span class="keyword">for</span> page <span class="keyword">in</span> <span class="built_in">range</span>(<span class="number">1</span>, <span class="number">5001</span>):</span><br><span class="line">        <span class="built_in">print</span>(<span class="string">&#x27;正在爬取第%d页的数据......&#x27;</span> % page)</span><br><span class="line">        <span class="comment"># 生成不同页码对应的url</span></span><br><span class="line">        url = <span class="string">&#x27;https://wz.sun0769.com/political/index/politicsNewest?id=1&amp;page=%d&#x27;</span> % page</span><br><span class="line">        page_text = requests.get(url=url, headers=headers, proxies=random.choice(proxy_list)).text</span><br><span class="line">        time.sleep(<span class="number">0.5</span>)</span><br><span class="line">        tree = etree.HTML(page_text)</span><br><span class="line">        <span class="comment"># 随便解析标题内容</span></span><br><span class="line">        li_list = tree.xpath(<span class="string">&#x27;/html/body/div[2]/div[3]/ul[2]/li&#x27;</span>)</span><br><span class="line">        <span class="keyword">for</span> li <span class="keyword">in</span> li_list:</span><br><span class="line">            title = li.xpath(<span class="string">&#x27;./span[3]/a/text()&#x27;</span>)[<span class="number">0</span>]</span><br><span class="line">            <span class="built_in">print</span>(title)</span><br></pre></td></tr></table></figure>
<h2 id="使用示例"><a href="#使用示例" class="headerlink" title="使用示例"></a>使用示例</h2><figure class="highlight python"><table><tr><td class="gutter"><pre><span class="line">1</span><br><span class="line">2</span><br><span class="line">3</span><br><span class="line">4</span><br><span class="line">5</span><br><span class="line">6</span><br><span class="line">7</span><br><span class="line">8</span><br><span class="line">9</span><br><span class="line">10</span><br><span class="line">11</span><br><span class="line">12</span><br><span class="line">13</span><br><span class="line">14</span><br><span class="line">15</span><br></pre></td><td class="code"><pre><span class="line"><span class="keyword">import</span> requests</span><br><span class="line"></span><br><span class="line"><span class="keyword">def</span> <span class="title function_">get</span>():</span><br><span class="line">    <span class="comment"># 1.指定url</span></span><br><span class="line">    main_url = <span class="string">&#x27;https://www.gushiwenwang.com/&#x27;</span></span><br><span class="line">    <span class="comment"># 2.发起请求:</span></span><br><span class="line">    response = requests.get(url=main_url)</span><br><span class="line">    <span class="comment"># 3.获取响应数据</span></span><br><span class="line">    page_text = response.text  <span class="comment"># text是可以返回字符串形式的响应数据/爬取到的数据</span></span><br><span class="line">    <span class="comment"># 4.持久化存储</span></span><br><span class="line">    <span class="keyword">with</span> <span class="built_in">open</span>(<span class="string">&#x27;gushiwenwang.html&#x27;</span>, <span class="string">&#x27;w&#x27;</span>) <span class="keyword">as</span> fp:</span><br><span class="line">        fp.write(page_text)</span><br><span class="line"></span><br><span class="line"><span class="keyword">if</span> __name__ == <span class="string">&#x27;__main__&#x27;</span>:</span><br><span class="line">    get()</span><br></pre></td></tr></table></figure>
</article><div class="post-copyright"><div class="post-copyright__author"><span class="post-copyright-meta"><i class="fas fa-circle-user fa-fw"></i>文章作者: </span><span class="post-copyright-info"><a href="https://houyudong1989.gitee.io/blog">侯玉东</a></span></div><div class="post-copyright__type"><span class="post-copyright-meta"><i class="fas fa-square-arrow-up-right fa-fw"></i>文章链接: </span><span class="post-copyright-info"><a href="https://houyudong1989.gitee.io/blog/2024/04/12/python/11%E3%80%81requests%E6%A8%A1%E5%9D%97%E4%BD%BF%E7%94%A8/">https://houyudong1989.gitee.io/blog/2024/04/12/python/11%E3%80%81requests%E6%A8%A1%E5%9D%97%E4%BD%BF%E7%94%A8/</a></span></div><div class="post-copyright__notice"><span class="post-copyright-meta"><i class="fas fa-circle-exclamation fa-fw"></i>版权声明: </span><span class="post-copyright-info">本博客所有文章除特别声明外，均采用 <a href="https://creativecommons.org/licenses/by-nc-sa/4.0/" target="_blank">CC BY-NC-SA 4.0</a> 许可协议。转载请注明来自 <a href="https://houyudong1989.gitee.io/blog" target="_blank">East'blog</a>！</span></div></div><div class="tag_share"><div class="post-meta__tag-list"><a class="post-meta__tags" href="/blog/tags/requests%E6%A8%A1%E5%9D%97%E4%BD%BF%E7%94%A8/">requests模块使用</a></div><div class="post_share"><div class="social-share" data-image="/blog/img/library1.jpeg" data-sites="facebook,twitter,wechat,weibo,qq"></div><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/butterfly-extsrc/sharejs/dist/css/share.min.css" media="print" onload="this.media='all'"><script src="https://cdn.jsdelivr.net/npm/butterfly-extsrc/sharejs/dist/js/social-share.min.js" defer></script></div></div><nav class="pagination-post" id="pagination"><div class="prev-post pull-left"><a href="/blog/2024/04/12/python/12%E3%80%81UA%E5%92%8CReferer%E5%8F%8D%E7%88%AC/" title="UA和Referer反爬"><img class="cover" src="/blog/img/library1.jpeg" onerror="onerror=null;src='/blog/img/friend_404.gif'" alt="cover of previous post"><div class="pagination-info"><div class="label">上一篇</div><div class="prev_info">UA和Referer反爬</div></div></a></div><div class="next-post pull-right"><a href="/blog/2024/04/12/python/10%E3%80%81%E7%88%AC%E8%99%AB%E4%BB%8B%E7%BB%8D/" title="爬虫介绍"><img class="cover" src="/blog/img/library1.jpeg" onerror="onerror=null;src='/blog/img/friend_404.gif'" alt="cover of next post"><div class="pagination-info"><div class="label">下一篇</div><div class="next_info">爬虫介绍</div></div></a></div></nav></div><div class="aside-content" id="aside-content"><div class="card-widget card-info"><div class="is-center"><div class="avatar-img"><img src="/blog/img/head.jpg" onerror="this.onerror=null;this.src='/blog/img/friend_404.gif'" alt="avatar"/></div><div class="author-info__name">侯玉东</div><div class="author-info__description">生命不息，奋斗不止</div></div><div class="card-info-data site-data is-center"><a href="/blog/archives/"><div class="headline">文章</div><div class="length-num">70</div></a><a href="/blog/tags/"><div class="headline">标签</div><div class="length-num">26</div></a><a href="/blog/categories/"><div class="headline">分类</div><div class="length-num">18</div></a></div><a id="card-info-btn" target="_blank" rel="noopener" href="https://gitee.com/houyudong1989"><i class="fab fa-github"></i><span>Follow Me</span></a><div class="card-info-social-icons is-center"><a class="social-icon" href="https://gitee.com/houyudong1989" target="_blank" title="Github"><i class="fab fa-github" style="color: #24292e;"></i></a><a class="social-icon" href="mailto:420098321@qq.com" target="_blank" title="Email"><i class="fas fa-envelope" style="color: #4a7dbe;"></i></a></div></div><div class="card-widget card-announcement"><div class="item-headline"><i class="fas fa-bullhorn fa-shake"></i><span>公告</span></div><div class="announcement_content">欢迎来到我的博客！！！</div></div><div class="sticky_layout"><div class="card-widget" id="card-toc"><div class="item-headline"><i class="fas fa-stream"></i><span>目录</span><span class="toc-percentage"></span></div><div class="toc-content"><ol class="toc"><li class="toc-item toc-level-1"><a class="toc-link" href="#requests%E6%A8%A1%E5%9D%97"><span class="toc-number">1.</span> <span class="toc-text">requests模块</span></a><ol class="toc-child"><li class="toc-item toc-level-2"><a class="toc-link" href="#requests-%E5%AE%89%E8%A3%85"><span class="toc-number">1.1.</span> <span class="toc-text">requests 安装</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#requests%E5%B8%B8%E7%94%A8%E6%96%B9%E6%B3%95"><span class="toc-number">1.2.</span> <span class="toc-text">requests常用方法</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#response-%E5%B8%B8%E7%94%A8%E5%87%BD%E6%95%B0"><span class="toc-number">1.3.</span> <span class="toc-text">response 常用函数</span></a></li><li class="toc-item toc-level-2"><a class="toc-link" href="#%E4%BB%A3%E7%90%86"><span class="toc-number">1.4.</span> <span class="toc-text">代理</span></a><ol class="toc-child"><li class="toc-item toc-level-3"><a class="toc-link" href="#%E8%AE%BF%E9%97%AE%E5%A6%82%E4%B8%8B%E7%BD%91%E5%9D%80%EF%BC%8C%E8%BF%94%E5%9B%9E%E8%87%AA%E5%B7%B1%E6%9C%AC%E6%9C%BAip"><span class="toc-number">1.4.1.</span> <span class="toc-text">访问如下网址，返回自己本机ip</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E4%BD%BF%E7%94%A8%E4%BB%A3%E7%90%86%E5%8F%91%E8%B5%B7%E8%AF%B7%E6%B1%82%EF%BC%8C%E6%9F%A5%E7%9C%8B%E6%98%AF%E5%90%A6%E5%8F%AF%E4%BB%A5%E8%BF%94%E5%9B%9E%E4%BB%A3%E7%90%86%E6%9C%8D%E5%8A%A1%E5%99%A8%E7%9A%84ip"><span class="toc-number">1.4.2.</span> <span class="toc-text">使用代理发起请求，查看是否可以返回代理服务器的ip</span></a></li><li class="toc-item toc-level-3"><a class="toc-link" href="#%E6%9E%84%E5%BB%BA%E4%BB%A3%E7%90%86%E6%B1%A0%E8%BF%9B%E8%A1%8C%E8%AF%B7%E6%B1%82-%E9%9C%80%E8%A6%81%E7%94%B3%E8%AF%B7%E4%B8%89%E6%96%B9%E4%BB%A3%E7%90%86%E6%B1%A0"><span class="toc-number">1.4.3.</span> <span class="toc-text">构建代理池进行请求(需要申请三方代理池)</span></a></li></ol></li><li class="toc-item toc-level-2"><a class="toc-link" href="#%E4%BD%BF%E7%94%A8%E7%A4%BA%E4%BE%8B"><span class="toc-number">1.5.</span> <span class="toc-text">使用示例</span></a></li></ol></li></ol></div></div><div class="card-widget card-recent-post"><div class="item-headline"><i class="fas fa-history"></i><span>最新文章</span></div><div class="aside-list"><div class="aside-list-item"><a class="thumbnail" href="/blog/2024/05/06/c++/5%E3%80%81%E5%AD%97%E7%AC%A6%E5%92%8C%E5%AD%97%E7%AC%A6%E4%B8%B2/" title="字符和字符串"><img src="/blog/img/library1.jpeg" onerror="this.onerror=null;this.src='/blog/img/friend_404.gif'" alt="字符和字符串"/></a><div class="content"><a class="title" href="/blog/2024/05/06/c++/5%E3%80%81%E5%AD%97%E7%AC%A6%E5%92%8C%E5%AD%97%E7%AC%A6%E4%B8%B2/" title="字符和字符串">字符和字符串</a><time datetime="2024-05-06T11:15:27.280Z" title="发表于 2024-05-06 19:15:27">2024-05-06</time></div></div><div class="aside-list-item"><a class="thumbnail" href="/blog/2024/05/06/c++/4%E3%80%81%E6%8C%87%E9%92%88%E5%92%8C%E5%87%BD%E6%95%B0%E6%8C%87%E9%92%88/" title="指针和函数指针"><img src="/blog/img/library1.jpeg" onerror="this.onerror=null;this.src='/blog/img/friend_404.gif'" alt="指针和函数指针"/></a><div class="content"><a class="title" href="/blog/2024/05/06/c++/4%E3%80%81%E6%8C%87%E9%92%88%E5%92%8C%E5%87%BD%E6%95%B0%E6%8C%87%E9%92%88/" title="指针和函数指针">指针和函数指针</a><time datetime="2024-05-06T11:14:29.047Z" title="发表于 2024-05-06 19:14:29">2024-05-06</time></div></div><div class="aside-list-item"><a class="thumbnail" href="/blog/2024/05/05/python/7%E3%80%81m3u8%E8%A7%86%E9%A2%91%E6%A0%BC%E5%BC%8F/" title="m3u8格式"><img src="/blog/img/library1.jpeg" onerror="this.onerror=null;this.src='/blog/img/friend_404.gif'" alt="m3u8格式"/></a><div class="content"><a class="title" href="/blog/2024/05/05/python/7%E3%80%81m3u8%E8%A7%86%E9%A2%91%E6%A0%BC%E5%BC%8F/" title="m3u8格式">m3u8格式</a><time datetime="2024-05-05T01:43:26.865Z" title="发表于 2024-05-05 09:43:26">2024-05-05</time></div></div><div class="aside-list-item"><a class="thumbnail" href="/blog/2024/04/30/android/charles%E5%AE%89%E5%8D%93%E6%89%8B%E6%9C%BA%E6%8A%93%E5%8C%85/" title="charles 抓包"><img src="/blog/img/library1.jpeg" onerror="this.onerror=null;this.src='/blog/img/friend_404.gif'" alt="charles 抓包"/></a><div class="content"><a class="title" href="/blog/2024/04/30/android/charles%E5%AE%89%E5%8D%93%E6%89%8B%E6%9C%BA%E6%8A%93%E5%8C%85/" title="charles 抓包">charles 抓包</a><time datetime="2024-04-30T06:49:29.279Z" title="发表于 2024-04-30 14:49:29">2024-04-30</time></div></div><div class="aside-list-item"><a class="thumbnail" href="/blog/2024/04/26/python/13%E3%80%81playwright/" title="playwright"><img src="/blog/img/library1.jpeg" onerror="this.onerror=null;this.src='/blog/img/friend_404.gif'" alt="playwright"/></a><div class="content"><a class="title" href="/blog/2024/04/26/python/13%E3%80%81playwright/" title="playwright">playwright</a><time datetime="2024-04-26T07:30:36.069Z" title="发表于 2024-04-26 15:30:36">2024-04-26</time></div></div></div></div></div></div></main><footer id="footer"><div id="footer-wrap"><div class="copyright">&copy;2020 - 2024 By 侯玉东</div><div class="framework-info"><span>框架 </span><a target="_blank" rel="noopener" href="https://hexo.io">Hexo</a><span class="footer-separator">|</span><span>主题 </span><a target="_blank" rel="noopener" href="https://github.com/jerryc127/hexo-theme-butterfly">Butterfly</a></div></div></footer></div><div id="rightside"><div id="rightside-config-hide"><button id="readmode" type="button" title="阅读模式"><i class="fas fa-book-open"></i></button><button id="darkmode" type="button" title="浅色和深色模式转换"><i class="fas fa-adjust"></i></button><button id="hide-aside-btn" type="button" title="单栏和双栏切换"><i class="fas fa-arrows-alt-h"></i></button></div><div id="rightside-config-show"><button id="rightside-config" type="button" title="设置"><i class="fas fa-cog fa-spin"></i></button><button class="close" id="mobile-toc-button" type="button" title="目录"><i class="fas fa-list-ul"></i></button><button id="go-up" type="button" title="回到顶部"><span class="scroll-percent"></span><i class="fas fa-arrow-up"></i></button></div></div><div><script src="/blog/js/utils.js"></script><script src="/blog/js/main.js"></script><script src="https://cdn.jsdelivr.net/npm/@fancyapps/ui/dist/fancybox/fancybox.umd.min.js"></script><div class="js-pjax"></div><div class="aplayer no-destroy" data-id="8873950678" data-server="netease" data-type="playlist" data-fixed="true" data-autoplay="true"> </div><script defer="defer" id="ribbon" src="https://cdn.jsdelivr.net/npm/butterfly-extsrc/dist/canvas-ribbon.min.js" size="150" alpha="0.6" zIndex="-1" mobile="false" data-click="false"></script><script defer="defer" id="fluttering_ribbon" mobile="false" src="https://cdn.jsdelivr.net/npm/butterfly-extsrc/dist/canvas-fluttering-ribbon.min.js"></script><script id="click-heart" src="https://cdn.jsdelivr.net/npm/butterfly-extsrc/dist/click-heart.min.js" async="async" mobile="false"></script><link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/aplayer/dist/APlayer.min.css" media="print" onload="this.media='all'"><script src="https://cdn.jsdelivr.net/npm/aplayer/dist/APlayer.min.js"></script><script src="https://cdn.jsdelivr.net/npm/butterfly-extsrc/metingjs/dist/Meting.min.js"></script><script src="https://cdn.jsdelivr.net/npm/pjax/pjax.min.js"></script><script>let pjaxSelectors = ["head > title","#config-diff","#body-wrap","#rightside-config-hide","#rightside-config-show",".js-pjax"]

var pjax = new Pjax({
  elements: 'a:not([target="_blank"])',
  selectors: pjaxSelectors,
  cacheBust: false,
  analytics: false,
  scrollRestoration: false
})

document.addEventListener('pjax:send', function () {

  // removeEventListener
  btf.removeGlobalFnEvent('pjax')
  btf.removeGlobalFnEvent('themeChange')

  document.getElementById('rightside').classList.remove('rightside-show')
  
  if (window.aplayers) {
    for (let i = 0; i < window.aplayers.length; i++) {
      if (!window.aplayers[i].options.fixed) {
        window.aplayers[i].destroy()
      }
    }
  }

  typeof typed === 'object' && typed.destroy()

  //reset readmode
  const $bodyClassList = document.body.classList
  $bodyClassList.contains('read-mode') && $bodyClassList.remove('read-mode')

  typeof disqusjs === 'object' && disqusjs.destroy()
})

document.addEventListener('pjax:complete', function () {
  window.refreshFn()

  document.querySelectorAll('script[data-pjax]').forEach(item => {
    const newScript = document.createElement('script')
    const content = item.text || item.textContent || item.innerHTML || ""
    Array.from(item.attributes).forEach(attr => newScript.setAttribute(attr.name, attr.value))
    newScript.appendChild(document.createTextNode(content))
    item.parentNode.replaceChild(newScript, item)
  })

  GLOBAL_CONFIG.islazyload && window.lazyLoadInstance.update()

  typeof panguInit === 'function' && panguInit()

  // google analytics
  typeof gtag === 'function' && gtag('config', '', {'page_path': window.location.pathname});

  // baidu analytics
  typeof _hmt === 'object' && _hmt.push(['_trackPageview',window.location.pathname]);

  typeof loadMeting === 'function' && document.getElementsByClassName('aplayer').length && loadMeting()

  // prismjs
  typeof Prism === 'object' && Prism.highlightAll()
})

document.addEventListener('pjax:error', e => {
  if (e.request.status === 404) {
    pjax.loadUrl('/blog/404.html')
  }
})</script><script async data-pjax src="//busuanzi.ibruce.info/busuanzi/2.3/busuanzi.pure.mini.js"></script></div></body></html>